# -*- coding:utf-8 -*-

import binascii
from module.ksw import Ksw
from module.store import Store

from multiprocessing.dummy import Pool as ThreadPool #多线程

complete = []

def main():

    # 初始化数据库
    Store.init()

    ls = Ksw.getlist("http://www.00ksw.org/html/3/3260/")

    pool = ThreadPool()
    pool.map(handle, ls)
    pool.close()
    pool.join()

    if Store.save( complete ):
        print("采集完毕, 并且全部存储成功")
    else:
        print("存储失败!!!")

def handle(url):
    info = Ksw.getcontent(url)
    name = binascii.crc32(info["content"].encode())

    with open("./resource/%s.txt" % name, "w", encoding="utf-8") as f:
        f.write(info["content"])

    # 完成任务后, 添加到列表
    complete.append( (info["title"], name) )

    print("%s-----抓取完毕" % info["title"])

if __name__ == "__main__":
    main()
